package com.li.spark

import org.apache.spark.{SparkConf, SparkContext}

/**
 * RDD创建的两种方式
 */
object Create2Ways {
  def main(args: Array[String]): Unit = {
    //创建SparkContext
    val conf = new SparkConf();
    conf.setAppName("Create2Ways").setMaster("local");
    val sc = new SparkContext(conf);

    //1.基于集合创建RDD
    val arr = Array(1, 2, 3, 4, 5)
    val rddArr = sc.parallelize(arr, 2)
    //求和
    val sum = rddArr.reduce(_ + _);
    println(sum);

    //2.基于HDFS生成RDD
    val rddHDFS = sc.textFile("hdfs://bigdata01:9000/hello_10000000.dat");
    println(rddHDFS.map(_.length).reduce(_ + _))

    sc.stop();
  }
}
